#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: detail_page_crawler.py
@time: 2018/1/5 11:27
"""
import random
import re
import time

from pyquery import PyQuery

from cache.record_cache import RecordCache
from common import util
from common.table_manage import TableManage
from config.app_conf import CRAWL_DETAIL_FAIL
from ext.crawler_handler import CrawlerHandler
from model.crawl_model import GsxtModel, STORE_TYPE_DETAIL, STORE_TYPE_LIST
from model.http_error import HttpError
from model.mq_model import MqModel
from model.primary_info import ENT_TYPE_TO_LAB


class DetailPageCrawler(CrawlerHandler):
    # url 映射字典
    GS_URL_EXTRACT_DICT = {
        GsxtModel.CONTRIBUTIVE_INFO: 'shareholderUrl',
        GsxtModel.KEY_PERSON_INFO: {"16": 'gtKeyPersonUrl',
                                    'other': 'keyPersonUrl'},
        GsxtModel.BRANCH_INFO: 'branchUrl',
        GsxtModel.CHANGE_INFO: {"16": 'gtAlertInfoUrl',
                                'other': 'alterInfoUrl'},
        GsxtModel.EQUITY_PLEDGED_INFO: 'stakQualitInfoUrl',
        GsxtModel.CHATTEL_MORTGAGE_INFO: 'mortRegInfoUrl',
        GsxtModel.SHAREHOLDER_INFO: 'insInvinfoUrl',
        # 知识产权出质登记信息
        # GsxtModel.INTELLECTUAL_PROPERTY_INFO: "proPledgeRegInfoUrl",
        # 商标注册信息（需要翻页）
        # GsxtModel.TRADEMARK_REGISTRATION_INFO: "trademarkInfoUrl",
        # 抽查检查结果信息
        # GsxtModel.SPOT_CHECK_RESULT_INFO: "spotCheckInfoUrl",
        # 双随机抽查检查结果信息
        # GsxtModel.DOUBLE_SPOT_CHECK_RESULT_INFO: "getDrRaninsResUrl",
        # 司法协助信息（特殊模块，变更和冻结来自2个url）
        # GsxtModel.JUDICIAL_ASSISTANCE_INFO: ["assistUrl", "judiciaryAltershareholderUrl"],
        # 官网提供行政许可信息
        # GsxtModel.ADMINISTRATIVE_LICENSING_INFO: "otherLicenceDetailInfoUrl",
        # 官网提供行政处罚信息
        GsxtModel.ADMINISTRATIVE_PENALTY_INFO: 'punishmentDetailInfoUrl',
        # 列入经营异常名录信息
        GsxtModel.ABNORMAL_OPERATION_INFO: {'other': 'entBusExcepUrl',
                                            '18': 'argBranchBusExcepUrl',
                                            '17': 'argBusExcepUrl',
                                            '16': 'indBusExcepUrl'},
        # 列入严重违法失信企业名单（黑名单）信息
        # GsxtModel.ILLEGAL_PROMISE_ENTERPRISE_INFO: "IllInfoUrl",
        # 股权变更信息
        # GsxtModel.CHANGE_SHAREHOLDING_INFO: "insAlterstockinfoUrl",
        # 企业即时行政许可信息（有详情页信息）
        # GsxtModel.COMPANY_ADMINISTRATIVE_LICENSING_INFO: "insLicenceinfoUrl",
        # 企业即时知识产权出质登记信息（有详情页信息）
        # GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO: "insProPledgeRegInfoUrl",
        # 企业即时行政许可信息（有详情页信息）
        # GsxtModel.COMPANY_ADMINISTRATIVE_PENALTY_INFO: "insLicenceinfoUrl",
        # # 失信被执行人信息（暂无案例，可能已取消该属性）
        # # GsxtModel.BREAK_PROMISE_INFO: "",
        # # 重大税收违法案件信息（暂无案例，可能已取消该属性）
        # # GsxtModel.TAX_ILLEGAL_INFO: "",
        # 清算信息
        # GsxtModel.LIQUIDATION_INFO: "liquidationUrl",
        # 简易注销公告信息
        # GsxtModel.SIMPLE_CANCER_INFO: "simpleCancelUrl",

        # 年报索引信息抽取
        # GsxtModel.ANNUAL_INFO: 'anCheYearInfo',
    }

    def __init__(self, parse_flag,
                 crawl_flag,
                 store_tube,
                 is_init_target_db,
                 is_init_source_db, log):
        super(DetailPageCrawler, self).__init__(store_tube, log,
                                                is_init_target_db=is_init_target_db,
                                                is_init_source_db=is_init_source_db)

        # 解析标识
        self.parse_flag = parse_flag

        # 抓取标识
        self.crawl_flag = crawl_flag
        self.crawl_flag_time = self.crawl_flag + '_time'

        self.log.info("详情页抓取对象初始化完成...")

    # 获得post详情页参数
    def __request_post_list(self, session, host, url_dict, page_model, field):
        url = url_dict.get(field)
        if url is None:
            return self.FAIL

        start = 0
        text_list = []
        current_total = 0
        draw = 1
        MAX_TRY_TIMES = 3
        is_success = True
        while True:
            try_times = 0
            recordsTotal = 0
            post_data = {
                'start': start,
                'draw': draw,
                'length': 5,
            }
            draw += 1
            while try_times < MAX_TRY_TIMES:

                resp, error = self.request_retry(session, host, session.post, url, data=post_data)
                if resp is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                if error == HttpError.NOT_FOUND:
                    self.log.error("当前属性不存在: field = {} url = {}".format(field, url))
                    return self.FAIL

                if resp.text == '':
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                json_data = util.json_loads(resp.text)
                if json_data is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                recordsTotal = json_data.get('recordsTotal')
                if recordsTotal is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                data = json_data.get('data')
                if not isinstance(data, list):
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                current = len(data)
                current_total += current
                text_list.append({'url': url, 'text': resp.text, 'post_data': post_data})
                break

            if try_times >= MAX_TRY_TIMES:
                is_success = False
                break

            if recordsTotal is None:
                is_success = False
                break

            if current_total >= recordsTotal:
                break

            start += 5

        # 如果抓取成功 或者抓到的数据不为空 则保留数据
        if is_success or len(text_list) > 0:
            # 存储属性
            page_model[self.get_store_field(field)] = {
                STORE_TYPE_LIST: text_list
            }

        if is_success:
            return self.SUCCESS

        return self.FAIL

    # 司法协助的特殊post请求
    def __request_post_judicial_list(self, session, host, url_dict, page_model, field):
        url = url_dict.get(field)
        if url is None:
            return self.FAIL

        # 司法协助信息列表结构为["assistUrl", "judiciaryAltershareholderUrl"]
        # 获取列表页信息使用"assistUrl"，故url[0]
        if isinstance(url, list):
            if len(url) == 2:
                url = url[0]
            else:
                return 0

        start = 0
        text_list = []
        current_total = 0
        draw = 1
        MAX_TRY_TIMES = 3
        is_success = True
        while True:
            try_times = 0
            recordsTotal = 0
            post_data = {
                'start': start,
                'draw': draw,
                'length': 5,
            }
            draw += 1
            while try_times < MAX_TRY_TIMES:

                resp, error = self.request_retry(session, host, session.post, url, data=post_data)
                if resp is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                if error == HttpError.NOT_FOUND:
                    self.log.error("当前属性不存在: field = {} url = {}".format(field, url))
                    return self.FAIL

                if resp.text == '':
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                json_data = util.json_loads(resp.text)
                if json_data is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                recordsTotal = json_data.get('recordsTotal')
                if recordsTotal is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                data = json_data.get('data')
                if not isinstance(data, list):
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                current = len(data)
                current_total += current
                text_list.append({'url': url, 'text': resp.text, 'post_data': post_data})
                break

            if try_times >= MAX_TRY_TIMES:
                is_success = False
                break

            if recordsTotal is None:
                is_success = False
                break

            if current_total >= recordsTotal:
                break

            start += 5

        # 如果抓取成功 或者抓到的数据不为空 则保留数据
        if is_success or len(text_list) > 0:
            # 存储属性
            page_model[self.get_store_field(field)] = {
                STORE_TYPE_LIST: text_list
            }

        if is_success:
            return self.SUCCESS

        return self.FAIL

    # 年报抓取请求，与通用get请求不同，数据结构不一样
    def __request_get_annual_list(self, session, host, url_dict, page_model, field):
        url = url_dict.get(field)
        if url is None:
            return self.FAIL

        def __inline_request_get_list(page_url):
            try_times = 0
            MAX_TRY_TIMES = 3
            while try_times < MAX_TRY_TIMES:

                resp, error = self.request_retry(session, host, session.get, page_url)
                if resp is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                if error == HttpError.NOT_FOUND:
                    return None

                json_data = util.json_loads(resp.text)
                if not isinstance(json_data, list):
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                return json_data

            return None

        # 请求第一页
        year_list = __inline_request_get_list(url)
        if year_list is None:
            return self.FAIL

        # 存储属性
        page_model[self.get_store_field(field)] = year_list
        return self.SUCCESS

    # get详情页参数
    def __request_get_list(self, session, host, url_dict, page_model, field):
        url = url_dict.get(field)
        if url is None:
            return self.FAIL

        text_data_list = []

        def __inline_request_get_list(page_url, text_list):
            try_times = 0
            records_total = 0
            current = 0
            MAX_TRY_TIMES = 3
            while try_times < MAX_TRY_TIMES:

                resp, error = self.request_retry(session, host, session.get, page_url)
                if resp is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                if error == HttpError.NOT_FOUND:
                    return None, None

                json_data = util.json_loads(resp.text)
                if json_data is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                records_total = json_data.get('recordsTotal')
                if records_total is None:
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                data = json_data.get('data')
                if not isinstance(data, list):
                    self.switch_proxy(session, host)
                    try_times += 1
                    continue

                current = len(data)
                text_list.append({'url': page_url, 'text': resp.text})
                break

            if try_times >= MAX_TRY_TIMES:
                return None, None

            return records_total, current

        # 请求第一页
        recordsTotal, currentNum = __inline_request_get_list(url, text_data_list)
        if recordsTotal is None:
            return self.FAIL

        # 继续翻页
        times = 1
        request_total = currentNum
        is_success = True
        while request_total < recordsTotal:
            start_url = url + '?' + 'start={}'.format(request_total)
            recordsTotal, currentNum = __inline_request_get_list(start_url, text_data_list)
            if recordsTotal is None:
                is_success = False
                break

            times += 1
            request_total += currentNum
            self.log.info("请求翻页次数: field = {} times = {} request_total = {} recordsTotal = {} currentNum = {}".format(
                field, times, request_total, recordsTotal, currentNum))

        # 如果是完全抓取成功，或者有数据则记录下来
        if is_success or len(text_data_list) > 0:
            # 存储属性
            page_model[self.get_store_field(field)] = {
                STORE_TYPE_LIST: text_data_list
            }

        if is_success:
            return self.SUCCESS

        return self.FAIL

    # 请求详情页
    def __request_detail(self, session, host, func, url, detail_list):
        try_times = 0
        MAX_TRY_TIMES = 3
        while try_times < MAX_TRY_TIMES:
            resp, error = self.request_retry(session, host, func, url)
            if resp is None:
                self.switch_proxy(session, host)
                try_times += 1
                continue

            if error == HttpError.NOT_FOUND:
                return self.FAIL

            result = util.json_loads(resp.text)
            if result is None:
                self.switch_proxy(session, host)
                try_times += 1
                continue

            if not isinstance(result.get('data'), list):
                self.switch_proxy(session, host)
                try_times += 1
                continue

            detail_list.append({'url': url, 'text': resp.text})
            break

        if try_times >= MAX_TRY_TIMES:
            return self.FAIL
        return self.SUCCESS

    # 请求企业即时信息的详情页
    def __request_company_detail(self, session, host, func, url, detail_list):
        try_times = 0
        MAX_TRY_TIMES = 3
        detail_url = ""
        # 获取拼接PROVINCENODENUM的url请求结果
        while try_times < MAX_TRY_TIMES:
            resp, error = self.request_retry(session, host, func, url)
            if resp is None:
                self.switch_proxy(session, host)
                try_times += 1
                continue

            if error == HttpError.NOT_FOUND:
                return self.FAIL

            res = util.json_loads(resp.text)
            if res is None:
                self.switch_proxy(session, host)
                try_times += 1
                continue
            # res 结构示例
            # {insLicenceDetailInfoUrl: "/corp-query-entprise-info-insLicenceDetailInfo-PROVINCENODENUM4500001994436804.html"}
            url_list = res.values()
            detail_url = "http://{}{}".format(host, url_list[0])
            break

        if try_times >= MAX_TRY_TIMES:
            return self.FAIL

        # 请求含有data的详情页
        try_times = 0
        while try_times < MAX_TRY_TIMES:
            resp, error = self.request_retry(session, host, func, detail_url)
            if resp is None:
                self.switch_proxy(session, host)
                try_times += 1
                continue

            if error == HttpError.NOT_FOUND:
                return self.FAIL

            result = util.json_loads(resp.text)
            if result is None:
                self.switch_proxy(session, host)
                try_times += 1
                continue

            if not isinstance(result.get('list'), list):
                self.switch_proxy(session, host)
                try_times += 1
                continue

            # resp 结构示例
            # {list: [ ]}
            detail_list.append({'url': url, 'text': resp.text})
            break

        if try_times >= MAX_TRY_TIMES:
            return self.FAIL
        return self.SUCCESS

    # 获取基本信息
    def get_base_info(self, session, host, href):
        session.headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,ja;q=0.6',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            'DNT': '1',
            'Host': host,
            'Referer': 'http://{}/corp-query-search-1.html'.format(host),
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': self.get_user_agent(),
        }
        url = 'http://{}{}'.format(host, href)
        resp, error = self.request_retry(session, host, session.get, url)
        if resp is None:
            return None, None

        if error == HttpError.NOT_FOUND:
            self.log.error("当前属性不存在: field = base_info url = {}".format(url))
            return None, None

        base_info = {
            STORE_TYPE_LIST: [{
                'url': url,
                'text': resp.text,
            }]
        }
        return resp.text, base_info

    # 获取详情页信息
    def __get_contributive_detail(self, session, host, page_model):
        # page_model[self.get_store_field(field)] = {
        #     STORE_TYPE_LIST: text_list
        # }
        field_info = self.get_store_field(GsxtModel.CONTRIBUTIVE_INFO)
        contributive_info = page_model.get(field_info)
        if not isinstance(contributive_info, dict):
            return self.FAIL

        text_list = contributive_info.get(STORE_TYPE_LIST)
        if not isinstance(text_list, list):
            return self.FAIL

        is_success = self.SUCCESS
        detail_list = []
        for item in text_list:
            text = item.get('text')
            json_data = util.json_loads(text)
            if json_data is None:
                is_success = self.FAIL
                continue

            data_list = json_data.get('data')
            if not isinstance(data_list, list):
                is_success = self.FAIL
                continue

            for json_item in data_list:
                # 判断是否有详情页
                detail_check = json_item.get('detailCheck')
                if detail_check != 'true':
                    continue

                inv_id = json_item.get('invId', '')
                detail_url = "http://{}/corp-query-entprise-info-shareholderDetail-{}.html".format(host, inv_id)
                is_success = self.__request_detail(session, host, session.get, detail_url, detail_list)

        # 存储属性
        page_model[field_info][STORE_TYPE_DETAIL] = detail_list
        return is_success

    # 抓取出资信息
    def get_contributive_info(self, session, host, company,
                              url_dict, page_model, gs_field_set):
        if GsxtModel.CONTRIBUTIVE_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.CONTRIBUTIVE_INFO)
        if is_success != self.SUCCESS:
            self.log.error("列表页抓取失败: company = {} field = {} cost time = {} s".format(
                company, GsxtModel.CONTRIBUTIVE_INFO, time.time() - start_time))
            return is_success

        is_success = self.__get_contributive_detail(session, host, page_model)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.CONTRIBUTIVE_INFO, is_success, time.time() - start_time))
        return is_success

    # 抓取主要人员
    def get_key_person_info(self, session, host, company,
                            url_dict, page_model, gs_field_set):
        if GsxtModel.KEY_PERSON_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.KEY_PERSON_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.KEY_PERSON_INFO, is_success, time.time() - start_time))
        return is_success

    # 抓取分支机构
    def get_branch_info(self, session, host, company,
                        url_dict, page_model, gs_field_set):
        if GsxtModel.BRANCH_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.BRANCH_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.BRANCH_INFO, is_success, time.time() - start_time))
        return is_success

    # 抓取变更信息
    def get_change_info(self, session, host, company,
                        url_dict, page_model, gs_field_set):
        if GsxtModel.CHANGE_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        session.headers['Origin'] = 'http://{}'.format(host)
        session.headers['Host'] = host
        session.headers['DNT'] = '1'
        session.headers['X-Requested-With'] = 'XMLHttpRequest'
        session.headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
        session.headers['Referer'] = url_dict[GsxtModel.BASE_INFO]
        session.headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'

        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.CHANGE_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.CHANGE_INFO, is_success, time.time() - start_time))
        return is_success

    # 抓取股权出质登记信息  测试案例 青海鑫诺光电科技有限公司
    def get_equity_pledged_info(self, session, host, company,
                                url_dict, page_model, gs_field_set):
        if GsxtModel.EQUITY_PLEDGED_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.EQUITY_PLEDGED_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.EQUITY_PLEDGED_INFO, is_success, time.time() - start_time))
        return is_success

    # 抓取动产抵押信息详情页
    def __get_chattel_mortgage_detail(self, session, host, page_model):
        field_info = self.get_store_field(GsxtModel.CHATTEL_MORTGAGE_INFO)
        equity_pledged_info = page_model.get(field_info)
        if not isinstance(equity_pledged_info, dict):
            return self.FAIL

        text_list = equity_pledged_info.get(STORE_TYPE_LIST)
        if not isinstance(text_list, list):
            return self.FAIL

        is_success = self.SUCCESS
        detail_list = []

        kew_word_list = ['mortregpersoninfo',
                         'mortCreditorRightInfo',
                         'mortGuaranteeInfo',
                         'getMortAltItemInfo',
                         'getMortRegCancelInfo']
        for item in text_list:
            text = item.get('text')
            json_data = util.json_loads(text)
            if json_data is None:
                is_success = self.FAIL
                continue

            data_list = json_data.get('data')
            if not isinstance(data_list, list):
                is_success = self.FAIL
                continue

            for json_item in data_list:
                # 动产抵押登记信息 详情页
                # http://qh.gsxt.gov.cn/corp-query-entprise-info-mortregpersoninfo-PROVINCENODENUM630000163000002263594429.html
                # http://qh.gsxt.gov.cn/corp-query-entprise-info-mortCreditorRightInfo-PROVINCENODENUM630000163000002263594429.html
                # http://qh.gsxt.gov.cn/corp-query-entprise-info-mortGuaranteeInfo-PROVINCENODENUM630000163000002263594429.html
                # http://qh.gsxt.gov.cn/corp-query-entprise-info-getMortAltItemInfo-PROVINCENODENUM630000163000002263594429.html
                # http://qh.gsxt.gov.cn/corp-query-entprise-info-getMortRegCancelInfo-PROVINCENODENUM630000163000002263594429.html
                # morReg_Id

                mor_reg_id = json_item.get('morReg_Id', '')
                if mor_reg_id is '':
                    continue

                # 按url关键字遍历
                for keyword in kew_word_list:
                    detail_url = "http://{}/corp-query-entprise-info-{}-{}.html".format(host, keyword, mor_reg_id)
                    is_success = self.__request_detail(session, host, session.get, detail_url, detail_list)

        # 存储属性
        page_model[field_info][STORE_TYPE_DETAIL] = detail_list
        return is_success

    # 抓取动产抵押信息 测试案例 青海鑫诺光电科技有限公司
    def get_chattel_mortgage_info(self, session, host, company,
                                  url_dict, page_model, gs_field_set):
        if GsxtModel.CHATTEL_MORTGAGE_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.CHATTEL_MORTGAGE_INFO)
        if is_success != self.SUCCESS:
            self.log.error("列表页抓取失败: company = {} field = {} cost time = {} s".format(
                company, GsxtModel.CHATTEL_MORTGAGE_INFO, time.time() - start_time))
            return is_success

        is_success = self.__get_chattel_mortgage_detail(session, host, page_model)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.CHATTEL_MORTGAGE_INFO, is_success, time.time() - start_time))
        return is_success

    # 抓取股东信息 年报下面那个
    def get_shareholder_info(self, session, host, company,
                             url_dict, page_model, gs_field_set):
        if GsxtModel.SHAREHOLDER_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.SHAREHOLDER_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.SHAREHOLDER_INFO, is_success, time.time() - start_time))

        return is_success

    # 抓取异常经营列表
    def get_abnormal_operation_info(self, session, host, company,
                                    url_dict, page_model, gs_field_set):
        if GsxtModel.ABNORMAL_OPERATION_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.ABNORMAL_OPERATION_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.ABNORMAL_OPERATION_INFO, is_success, time.time() - start_time))
        return is_success

    # 抓取行政处罚信息
    def get_administrative_penalty_info(self, session, host, company,
                                        url_dict, page_model, gs_field_set):
        if GsxtModel.ADMINISTRATIVE_PENALTY_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.ADMINISTRATIVE_PENALTY_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.ADMINISTRATIVE_PENALTY_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取知识产权出质登记信息
    def get_intellectual_property_info(self, session, host, company,
                                       url_dict, page_model, gs_field_set):
        if GsxtModel.INTELLECTUAL_PROPERTY_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.INTELLECTUAL_PROPERTY_INFO)
        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.INTELLECTUAL_PROPERTY_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取商标注册信息
    def get_trademark_registration_info(self, session, host, company,
                                        url_dict, page_model, gs_field_set):
        if GsxtModel.TRADEMARK_REGISTRATION_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.TRADEMARK_REGISTRATION_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.TRADEMARK_REGISTRATION_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取抽查检查结果信息
    def get_spot_check_result_info(self, session, host, company,
                                   url_dict, page_model, gs_field_set):
        if GsxtModel.SPOT_CHECK_RESULT_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.SPOT_CHECK_RESULT_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.SPOT_CHECK_RESULT_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取双随机抽查检查结果信息
    def get_double_spot_check_result_info(self, session, host, company,
                                          url_dict, page_model, gs_field_set):
        if GsxtModel.DOUBLE_SPOT_CHECK_RESULT_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.DOUBLE_SPOT_CHECK_RESULT_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.DOUBLE_SPOT_CHECK_RESULT_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取司法协助信息（特殊模块，数据来自两个url）
    def get_judicial_assistance_info(self, session, host, company,
                                     url_dict, page_model, gs_field_set):
        if GsxtModel.JUDICIAL_ASSISTANCE_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_judicial_list(session, host, url_dict,
                                                       page_model, GsxtModel.JUDICIAL_ASSISTANCE_INFO)
        if is_success != self.SUCCESS:
            self.log.error("列表页抓取失败: company = {} field = {} cost time = {} s".format(
                company, GsxtModel.JUDICIAL_ASSISTANCE_INFO, time.time() - start_time))
            return is_success

        # 特殊模块要用url_dict传递 股权变更 的详情页url
        is_success = self.__get_judicial_assistance_detail(session, host, url_dict, page_model)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.JUDICIAL_ASSISTANCE_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取司法协助信息详情页 （特殊模块，数据来自两个url）
    def __get_judicial_assistance_detail(self, session, host, url_dict, page_model):
        # page_model[self.get_store_field(field)] = {
        #     STORE_TYPE_LIST: text_list
        # }
        field_info = self.get_store_field(GsxtModel.JUDICIAL_ASSISTANCE_INFO)
        judicial_assistance_info = page_model.get(field_info)
        if not isinstance(judicial_assistance_info, dict):
            return self.FAIL

        text_list = judicial_assistance_info.get(STORE_TYPE_LIST)
        if not isinstance(text_list, list):
            return self.FAIL

        is_success = self.SUCCESS
        detail_list = []
        for item in text_list:
            text = item.get('text')
            json_data = util.json_loads(text)
            if json_data is None:
                is_success = self.FAIL
                continue

            data_list = json_data.get('data')
            if not isinstance(data_list, list):
                is_success = self.FAIL
                continue

            has_alter_shareholder = False
            for json_item in data_list:
                # type 为 1，表示"冻结／解除冻结"
                if json_item.get("type") == "1":
                    parent_id = json_item.get('parent_Id', '')
                    detail_url = "http://{}/corp-query-entprise-info-judiciaryStockfreeze-{}.html".format(host,
                                                                                                          parent_id)
                    is_success = self.__request_detail(session, host, session.get, detail_url, detail_list)
                # type 为 2，表示"股权变更"
                if json_item.get("type") == "2":
                    has_alter_shareholder = True

            # 判断是否有股权变更信息
            if has_alter_shareholder:
                # 司法协助存储的列表结构为[assistUrl, judiciaryAltershareholderUrl], 故取[1]
                detail_url = url_dict[GsxtModel.JUDICIAL_ASSISTANCE_INFO][1]
                is_success = self.__request_detail(session, host, session.get, detail_url, detail_list)

        # 存储属性
        page_model[field_info][STORE_TYPE_DETAIL] = detail_list
        return is_success

    # TODO 抓取官网提供行政许可信息
    def get_administrative_licensing_info(self, session, host, company,
                                          url_dict, page_model, gs_field_set):
        if GsxtModel.ADMINISTRATIVE_LICENSING_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.ADMINISTRATIVE_LICENSING_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.ADMINISTRATIVE_LICENSING_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取列入严重违法失信企业名单（黑名单）信息
    def get_illegal_promise_enterprise_info(self, session, host, company,
                                            url_dict, page_model, gs_field_set):
        if GsxtModel.ILLEGAL_PROMISE_ENTERPRISE_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.ILLEGAL_PROMISE_ENTERPRISE_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.ILLEGAL_PROMISE_ENTERPRISE_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取新增股权变更信息
    def get_change_shareholding_info(self, session, host, company,
                                     url_dict, page_model, gs_field_set):
        if GsxtModel.CHANGE_SHAREHOLDING_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.CHANGE_SHAREHOLDING_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.CHANGE_SHAREHOLDING_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取企业即时行政许可信息
    def get_company_administrative_licensing_info(self, session, host, company,
                                                  url_dict, page_model, gs_field_set):
        if GsxtModel.COMPANY_ADMINISTRATIVE_LICENSING_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.COMPANY_ADMINISTRATIVE_LICENSING_INFO)
        if is_success != self.SUCCESS:
            self.log.error("列表页抓取失败: company = {} field = {} cost time = {} s".format(
                company, GsxtModel.COMPANY_ADMINISTRATIVE_LICENSING_INFO, time.time() - start_time))
            return is_success

        is_success = self.__get_company_administrative_licensing_detail(session, host, page_model)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.COMPANY_ADMINISTRATIVE_LICENSING_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取企业即时行政许可信息详情页
    def __get_company_administrative_licensing_detail(self, session, host, page_model):
        # page_model[self.get_store_field(field)] = {
        #     STORE_TYPE_LIST: text_list
        # }
        field_info = self.get_store_field(GsxtModel.COMPANY_ADMINISTRATIVE_LICENSING_INFO)
        company_administrative_licensing_info = page_model.get(field_info)
        if not isinstance(company_administrative_licensing_info, dict):
            return self.FAIL

        text_list = company_administrative_licensing_info.get(STORE_TYPE_LIST)
        if not isinstance(text_list, list):
            return self.FAIL

        is_success = self.SUCCESS
        detail_list = []
        for item in text_list:
            text = item.get('text')
            json_data = util.json_loads(text)
            if json_data is None:
                is_success = self.FAIL
                continue

            data_list = json_data.get('data')
            if not isinstance(data_list, list):
                is_success = self.FAIL
                continue

            for json_item in data_list:
                # 当data_list为空列表，则直接跳出
                lic_id = json_item.get('licId', '')
                detail_url = "http://{}/corp-query-entprise-info-insLicenceInfoForJs-{}.html".format(host, lic_id)
                is_success = self.__request_company_detail(session, host, session.get, detail_url, detail_list)

        # 存储属性
        page_model[field_info][STORE_TYPE_DETAIL] = detail_list
        return is_success

    # TODO 抓取企业即时知识产权出质登记信息
    def get_company_intellectual_property_info(self, session, host, company,
                                               url_dict, page_model, gs_field_set):
        if GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO)
        if is_success != self.SUCCESS:
            self.log.error("列表页抓取失败: company = {} field = {} cost time = {} s".format(
                company, GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO, time.time() - start_time))
            return is_success

        is_success = self.__get_company_intellectual_property_detail(session, host, page_model)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取企业即时知识产权出质登记信息详情页
    def __get_company_intellectual_property_detail(self, session, host, page_model):
        # page_model[self.get_store_field(field)] = {
        #     STORE_TYPE_LIST: text_list
        # }
        field_info = self.get_store_field(GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO)
        company_intellectual_property_info = page_model.get(field_info)
        if not isinstance(company_intellectual_property_info, dict):
            return self.FAIL

        text_list = company_intellectual_property_info.get(STORE_TYPE_LIST)
        if not isinstance(text_list, list):
            return self.FAIL

        is_success = self.SUCCESS
        detail_list = []
        for item in text_list:
            text = item.get('text')
            json_data = util.json_loads(text)
            if json_data is None:
                is_success = self.FAIL
                continue

            data_list = json_data.get('data')
            if not isinstance(data_list, list):
                is_success = self.FAIL
                continue

            for json_item in data_list:
                # 当data_list为空列表，则直接跳出
                ple_id = json_item.get('pleId', '')
                detail_url = "http://{}/corp-query-entprise-info-insImKlpInfoForJs-{}.html".format(host, ple_id)
                is_success = self.__request_company_detail(session, host, session.get, detail_url, detail_list)

        # 存储属性
        page_model[field_info][STORE_TYPE_DETAIL] = detail_list
        return is_success

    # TODO 抓取企业即时行政处罚信息
    def get_company_administrative_penalty_info(self, session, host, company,
                                                url_dict, page_model, gs_field_set):
        if GsxtModel.COMPANY_ADMINISTRATIVE_PENALTY_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_post_list(session, host, url_dict,
                                              page_model, GsxtModel.COMPANY_ADMINISTRATIVE_PENALTY_INFO)
        if is_success != self.SUCCESS:
            self.log.error("列表页抓取失败: company = {} field = {} cost time = {} s".format(
                company, GsxtModel.COMPANY_ADMINISTRATIVE_PENALTY_INFO, time.time() - start_time))
            return is_success

        is_success = self.__get_company_administrative_penalty_detail(session, host, page_model)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取企业即时行政处罚信息详情页
    def __get_company_administrative_penalty_detail(self, session, host, page_model):
        # page_model[self.get_store_field(field)] = {
        #     STORE_TYPE_LIST: text_list
        # }
        field_info = self.get_store_field(GsxtModel.COMPANY_ADMINISTRATIVE_PENALTY_INFO)
        company_administrative_penalty_info = page_model.get(field_info)
        if not isinstance(company_administrative_penalty_info, dict):
            return self.FAIL

        text_list = company_administrative_penalty_info.get(STORE_TYPE_LIST)
        if not isinstance(text_list, list):
            return self.FAIL

        is_success = self.SUCCESS
        detail_list = []
        for item in text_list:
            text = item.get('text')
            json_data = util.json_loads(text)
            if json_data is None:
                is_success = self.FAIL
                continue

            data_list = json_data.get('data')
            if not isinstance(data_list, list):
                is_success = self.FAIL
                continue

            for json_item in data_list:
                # 当data_list为空列表，则直接跳出
                lic_id = json_item.get('licId', '')
                detail_url = "http://{}//corp-query-entprise-info-insPunshmentInfoForJs-{}.html".format(host, lic_id)
                is_success = self.__request_company_detail(session, host, session.get, detail_url, detail_list)

        # 存储属性
        page_model[field_info][STORE_TYPE_DETAIL] = detail_list
        return is_success

    # TODO 抓取失信被执行人信息
    def get_break_promise_info(self, session, host, company,
                               url_dict, page_model, gs_field_set):
        pass

    # TODO 抓取重大税收违法案件信息
    def get_tax_illegal_info(self, session, host, company,
                             url_dict, page_model, gs_field_set):
        pass

    # TODO 抓取清算信息
    def get_liquidation_info(self, session, host, company,
                             url_dict, page_model, gs_field_set):
        if GsxtModel.LIQUIDATION_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.LIQUIDATION_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.LIQUIDATION_INFO, is_success, time.time() - start_time))
        return is_success

    # TODO 抓取简易注销公告信息
    def get_simple_cancer_info(self, session, host, company,
                               url_dict, page_model, gs_field_set):
        if GsxtModel.SIMPLE_CANCER_INFO not in gs_field_set:
            return self.FAIL

        start_time = time.time()
        is_success = self.__request_get_list(session, host, url_dict,
                                             page_model, GsxtModel.SIMPLE_CANCER_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.SIMPLE_CANCER_INFO, is_success, time.time() - start_time))

    # 抽取基本链接与索引链接
    def select_annual_href(self, annual_type, ent_type, url_all_link_dict):
        base_href = None
        index_href = None
        try:
            ent_type = int(ent_type)
        except Exception as e:
            self.log.error("ent_type 类型转换错误: ent_type = {}".format(ent_type))
            self.log.exception(e)
            return base_href, index_href

        if annual_type == 'e' and 0 < ent_type < 18:
            base_href = url_all_link_dict.get('VAnnualReportBaseinfoUrl')
            index_href = url_all_link_dict.get('vAnnualReportBaseInfoForJsUrl')
        elif annual_type == 'sfc':
            base_href = url_all_link_dict.get('VAnnualReportSfcBaseinfoUrl')
            index_href = url_all_link_dict.get('vAnnualSfcReportBaseInfoForJsUrl')
        elif annual_type == 'pb':
            base_href = url_all_link_dict.get('VAnnualReportPbBaseinfoUrl')
            index_href = url_all_link_dict.get('vAnnualPbReportBaseInfoForJsUrl')
        else:
            base_href = url_all_link_dict.get('VAnnualReportBaseinfoUrl')
            index_href = url_all_link_dict.get('vAnnualReportBaseInfoForJsUrl')

        return base_href, index_href

    # 处理年报
    def process_annual(self, province, company, ent_type, url_all_link_dict, annual_list):
        # {
        #     # 年报关键ID
        #     '_id': anCheId,
        #     # 解析出来的企业名称
        #     'company': parse_comany,
        #     'year': 2014,
        #     # 企业类型，不是必要的但需要保存
        #     'ent_type': 18
        #     # 年报类型，不是必要的但需要保存
        #     'annual_type': 'e',
        #     # 通过 ent_type 与 annual_type 计算出来的
        #     'base_href': xxx,
        #     # 通过 ent_type 与 annual_type 计算出来的
        #     'index_href': xxxx,
        #     '_in_time': xxx
        #     'crawl_status': 0, 1, 2
        #     'crawl_status_time': '2018-01-23'
        # }
        # 需要存储的表名
        annual_param_table = TableManage.get_annual_table(province)
        for annual_item in annual_list:

            _id = annual_item.get('anCheId')
            if _id is None:
                continue

            year = annual_item.get('anCheYear')
            if year is None:
                continue

            annual_type = annual_item.get('entType')
            base_href, index_href = self.select_annual_href(annual_type, ent_type, url_all_link_dict)
            if base_href is None or index_href is None:
                continue

            item = {
                '_id': _id,
                'company': company,
                'year': year,
                'ent_type': ent_type,
                'annual_type': annual_type,
                'base_href': base_href,
                'index_href': index_href,
                '_in_time': util.get_now_time(),
            }

            # 存储年报索引信息
            self.save_seed_info(annual_param_table, item)

    # 抓取年报索引信息
    def get_annual_info(self, session, host, company, url_dict, page_model, gs_field_set):
        if GsxtModel.ANNUAL_INFO not in gs_field_set:
            return self.FAIL

        session.headers['Host'] = host
        session.headers['DNT'] = '1'
        session.headers['X-Requested-With'] = 'XMLHttpRequest'
        session.headers['Accept'] = '*/*'
        # r = session.get('http://cnzz.mmstat.com/9.gif?abc=1&rnd=2758692')
        # r = session.get('http://pcookie.cnzz.com/app.gif?&cna=5b/XEo3TZ0ICAXkju0g2zZwA')
        # r = session.get('http://{}/SearchItemCaptcha?t={}'.format(host, util.get_time_stamp()))

        # 这里是实际抓取请求
        start_time = time.time()

        is_success = self.__request_get_annual_list(session, host, url_dict,
                                                    page_model, GsxtModel.ANNUAL_INFO)

        self.log.info("属性抓取状态: company = {} field = {} is_success = {} cost time = {} s".format(
            company, GsxtModel.ANNUAL_INFO, is_success, time.time() - start_time))
        return is_success

    # 抽取所有的链接
    @staticmethod
    def __get_all_url_link(text):
        url_link_dict = {}
        pattern = 'var (.*?)\s*=\s*"(.*?)"'
        regex = re.compile(pattern)
        search_list = regex.findall(text)
        for item in search_list:
            if len(item) < 2:
                continue

            if not isinstance(item[0], basestring):
                continue

            key = item[0].strip()
            if not isinstance(item[1], basestring):
                continue

            value = item[1].strip()
            url_link_dict[key] = value

        return url_link_dict

    # 抽取工商相关所有链接
    def __get_gs_url_link(self, host, ent_type, url_all_link_dict):
        url_link_dict = {}
        for field, value in self.GS_URL_EXTRACT_DICT.iteritems():
            if isinstance(value, list):
                # 司法协助模块需要传入2个url, 使用列表传递
                url_link_dict[field] = []
                for field_key in value:
                    if field_key in url_all_link_dict:
                        field_value = 'http://{}{}'.format(host, url_all_link_dict[field_key])
                        url_link_dict[field].append(field_value)
                continue

            if isinstance(value, basestring):
                field_key = value
            else:
                if ent_type in value:
                    field_key = value[ent_type]
                else:
                    field_key = value['other']

            if field_key in url_all_link_dict:
                url_link_dict[field] = 'http://{}{}'.format(host, url_all_link_dict[field_key])

        return url_link_dict

    # 获得url链接
    def get_company_and_ent_type(self, text):

        try:
            if text is None or text == '':
                self.log.error("企业名称解析文本不正常: text = {}".format(text))
                return None, None

            jq = PyQuery(text, parser='html')

            # 提取企业名称
            company = jq.find('.companyName').find('.fullName').text()
            if company is None or company == '':
                self.log.error("企业名称解析失败: text = {}".format(text))
                return None, None

            ent_type = jq.find('.container1').find('#entType').attr('value')
            if ent_type is None or ent_type == '':
                return None, None

        except Exception as e:
            self.log.error("解析公司名称失败: text = {}".format(text))
            self.log.exception(e)
            return None, None

        return ent_type, company

    # 获得存储属性
    @staticmethod
    def get_store_field(field):
        return '__' + field

    # 获得所有需要抓取的板块信息
    def get_gs_field_set(self, ent_type):
        full_field_dict = ENT_TYPE_TO_LAB.get(ent_type)
        if full_field_dict is None:
            self.log.error("当前ent_type = {} 没有对应的板块映射表...")
            return set(self.GS_URL_EXTRACT_DICT.keys())

        gs_field_set = set()
        for field, value in self.GS_URL_EXTRACT_DICT.iteritems():
            if field in full_field_dict:
                gs_field_set.add(field)

        return gs_field_set

    # 工商属性抓取入口
    def process_gsxt_field(self, session, host, parse_company,
                           url_all_link_dict, url_dict,
                           page_model, gs_field_set,
                           record_cache_dict):
        success_count = 0

        # 工商函数列表
        gs_func_dict = {
            GsxtModel.CONTRIBUTIVE_INFO: self.get_contributive_info,
            GsxtModel.KEY_PERSON_INFO: self.get_key_person_info,
            GsxtModel.BRANCH_INFO: self.get_branch_info,
            GsxtModel.CHANGE_INFO: self.get_change_info,
            GsxtModel.EQUITY_PLEDGED_INFO: self.get_equity_pledged_info,
            GsxtModel.CHATTEL_MORTGAGE_INFO: self.get_chattel_mortgage_info,
            GsxtModel.SHAREHOLDER_INFO: self.get_shareholder_info,
            # TODO 知识产权出质登记信息
            GsxtModel.INTELLECTUAL_PROPERTY_INFO: self.get_intellectual_property_info,
            # TODO 商标注册信息
            GsxtModel.TRADEMARK_REGISTRATION_INFO: self.get_trademark_registration_info,
            # TODO 抽查检查信息
            GsxtModel.SPOT_CHECK_RESULT_INFO: self.get_spot_check_result_info,
            # TODO 双随机抽查检查信息
            GsxtModel.DOUBLE_SPOT_CHECK_RESULT_INFO: self.get_double_spot_check_result_info,
            # TODO 司法协助信息
            GsxtModel.JUDICIAL_ASSISTANCE_INFO: self.get_judicial_assistance_info,
            # TODO 官网提供行政许可信息
            GsxtModel.ADMINISTRATIVE_LICENSING_INFO: self.get_administrative_licensing_info,
            # 官网提供行政处罚信息
            GsxtModel.ADMINISTRATIVE_PENALTY_INFO: self.get_administrative_penalty_info,
            # 列入经营异常名录信息
            GsxtModel.ABNORMAL_OPERATION_INFO: self.get_abnormal_operation_info,
            # TODO 列入严重违法失信企业名单（黑名单）信息
            GsxtModel.ILLEGAL_PROMISE_ENTERPRISE_INFO: self.get_illegal_promise_enterprise_info,
            # TODO 股权变更信息
            GsxtModel.CHANGE_SHAREHOLDING_INFO: self.get_change_shareholding_info,
            # TODO 企业即时行政许可信息
            GsxtModel.COMPANY_ADMINISTRATIVE_LICENSING_INFO: self.get_company_administrative_licensing_info,
            # TODO 企业即时知识产权出质登记信息
            GsxtModel.COMPANY_INTELLECTUAL_PROPERTY_INFO: self.get_company_intellectual_property_info,
            # TODO 企业即时行政处罚信息
            GsxtModel.COMPANY_ADMINISTRATIVE_PENALTY_INFO: self.get_company_administrative_penalty_info,
            # TODO 失信被执行人信息
            GsxtModel.BREAK_PROMISE_INFO: self.get_break_promise_info,
            # TODO 重大税收违法案件信息
            GsxtModel.TAX_ILLEGAL_INFO: self.get_tax_illegal_info,
            # TODO 清算信息
            GsxtModel.LIQUIDATION_INFO: self.get_liquidation_info,
            # TODO 简易注销公告信息
            GsxtModel.SIMPLE_CANCER_INFO: self.get_simple_cancer_info,
            # 年报索引抓取
            GsxtModel.ANNUAL_INFO: self.get_annual_info,
        }

        # sessionid 设置
        for _ in xrange(3):
            url = 'http://{}/SearchItemCaptcha?t={}'.format(host, util.get_time_stamp())
            r = session.get(url, timeout=10)
            url = 'http://{}/corp-query-custom-geetest-image.gif?v={}'.format(host, random.randint(1, 100))
            r = session.get(url, timeout=10)

        # 访问 getAttentionEffectiveUrl
        href = url_all_link_dict.get('getAttentionEffectiveUrl')
        if href is not None:
            url = 'http://{}{}'.format(host, href)
            r = session.post(url, timeout=10)

        for field, func in gs_func_dict.iteritems():

            # 如果当前属性并没有在控制范围内，则不抓取
            if field not in gs_field_set:
                continue

            # 判断是否需要抓取
            is_need_crawl = False
            while True:
                # 判断属性是否在缓存结构中
                if field not in record_cache_dict:
                    is_need_crawl = True
                    break

                # 获得上一次计算的时间
                cache_time = record_cache_dict[field]

                # 如果过期了
                if RecordCache.is_gs_expired(field, cache_time):
                    is_need_crawl = True
                    break
                break

            # 如果在缓存期间的话就不需要抓取，则统计+1
            if not is_need_crawl:
                success_count += 1
                continue

            is_success = func(session, host, parse_company, url_dict, page_model, gs_field_set)
            success_count += is_success
            # 如果抓取成功 则记录到缓存中
            if is_success == self.SUCCESS:
                record_cache_dict[field] = int(time.time())

        return success_count

    # 存储网页信息
    def store_web_page(self, province, company, status, seed_table, record_cache_dict, web_page_model):
        # 创建消息队列发送对象
        mq_model = MqModel()

        # 存储网页库信息
        mq_model.append_detail_page(TableManage.get_detail_page_table(province), web_page_model)

        # 针对只完成部分抓取的调度请求，也需要记录抓取状态以及最后抓取时间
        # if status == CRAWL_DETAIL_SUCCESS:
        seed_model = {
            '_id': company,
            self.crawl_flag: status,
            self.crawl_flag_time: util.get_now_time(),
        }
        mq_model.append_seed(seed_table, seed_model)

        # 缓冲控制 存储缓存信息
        mq_model.append_gs_cache(RecordCache.get_gs_cache_key(province, company),
                                 RecordCache.gs_record_to_cache(record_cache_dict))

        # 这里发送到存储消息队列
        self.send_store_beanstalk(mq_model.get_store_model())

    # 抓取工商属性
    def crawl_field(self, company, province, host, seed_table, href, record_cache_dict):

        # 初始化session
        session = self.get_session(host)

        # 抓取基本信息
        text, base_info = self.get_base_info(session, host, href)
        if text is None:
            self.log.error("基本信息抓取失败: province:{} host:{} company:{}".format(
                province, host, company))
            return CRAWL_DETAIL_FAIL

        # 获得链接信息
        ent_type, parse_company = self.get_company_and_ent_type(text)
        if parse_company is None:
            self.log.error("抽取关键信息失败: province:{} host:{} company:{}".format(
                province, host, company))
            return CRAWL_DETAIL_FAIL

        # 抽取所有的链接信息
        url_all_link_dict = self.__get_all_url_link(text)

        # 获取工商需要的链接信息
        gs_url_dict = self.__get_gs_url_link(host, ent_type, url_all_link_dict)

        # 添加基本信息链接到字典
        gs_url_dict[GsxtModel.BASE_INFO] = 'http://{}{}'.format(host, href)

        # 获得所有需要抓取的板块信息
        gs_field_set = self.get_gs_field_set(ent_type)

        # 创建存储对象
        web_page_model = {
            '_id': parse_company,
            '_in_time': util.get_now_time(),
            'province': province,
            # 重置解析标识
            self.parse_flag: 0,
            self.get_store_field(GsxtModel.BASE_INFO): base_info,
        }

        # 处理工商属性抓取
        gs_success_count = self.process_gsxt_field(session, host, parse_company,
                                                   url_all_link_dict, gs_url_dict, web_page_model,
                                                   gs_field_set, record_cache_dict)

        # 获取抓取状态信息
        status = self.get_crawl_status(gs_success_count, len(gs_field_set))

        # 如果为所有属性都抓取失败状态，则不存储任何状态信息
        if status == CRAWL_DETAIL_FAIL:
            return status

        # 处理年报
        if self.get_store_field(GsxtModel.ANNUAL_INFO) in web_page_model:
            self.process_annual(province, parse_company,
                                ent_type, url_all_link_dict,
                                web_page_model[self.get_store_field(GsxtModel.ANNUAL_INFO)])

        # 存储抓取的网页信息
        self.store_web_page(province, company, status, seed_table, record_cache_dict, web_page_model)
        return status

    # 抓取详情页
    def crawl_process(self, process_id, param_dict, record_cache_dict):
        # 'company': company,
        # 'href': href,
        # # 这里的host 与省份没有直接关系，比如 gs.gsxt.gov.cn 搜出安徽的企业详情页信息，因为搜索的是全国
        # 'host': host,
        # 'province': province,

        # 基本信息链接
        href = param_dict.get('href')
        if href is None:
            return CRAWL_DETAIL_FAIL

        # 需要抓取的域名
        host = param_dict.get('host')
        if host is None:
            return CRAWL_DETAIL_FAIL

        # 网页库表 根据省份进行计算
        province = param_dict.get('province')

        # 企业名称，列表页搜索出来的 search_name 字段
        company = param_dict.get('company')

        # # 测试 南京药石科技股份有限公司
        # href = '/%7Bwpwa8Db2SThwaF07n77HycxC6uhfnHnuBaEaA2J6A1Bi9Qok1LPm-17zUYPfeaEQ69FafEcjBP_9D2lwOpHks3y_FFd9EBtCiJt1-vKHg9zTp4VZ8vW3zPFu0tE5FLXy6T1OdBNNpVMyII1vGZcK5DQRUXvp1gBBMebfrQPFl8w-1517407634419%7D'
        # host = 'js.gsxt.gov.cn'
        # province = 'jiangsu'
        # company = '南京药石科技股份有限公司'

        # 获得来源表信息
        seed_table = param_dict.get('seed_table')
        if seed_table is None:
            seed_table = TableManage.get_seed_table(province)

        self.log.info("当前详情页抓取: process_id:{} province:{} host:{} company:{}".format(
            process_id, province, host, company))

        # 抓取工商属性
        try:
            status = self.crawl_field(company, province, host, seed_table, href,
                                      record_cache_dict)
        except Exception as e:
            status = CRAWL_DETAIL_FAIL
            self.log.error("抓取异常: process_id:{} province:{} host:{} status:{} company:{}".format(
                process_id, province, host, status, company))
            self.log.exception(e)

        self.log.info("详情页抓取状态: process_id:{} province:{} host:{} status:{} company:{}".format(
            process_id, province, host, status, company))

        return status
